# # from selenium import webdriver
# # from selenium.webdriver.common.by import By
# # from selenium.webdriver.edge.service import Service  # 导入Edge服务模块
# # from beautifultable import BeautifulTable
# # from selenium.webdriver.support.ui import WebDriverWait
# # from selenium.webdriver.support import expected_conditions as EC
# # from selenium.common.exceptions import NoSuchElementException, TimeoutException
# # import time
# # driver = webdriver.Edge(service=Service())
# #
# # def get_html(url,max_page):
# #     driver.get(url)
# #     data=[]
# #     current_page = 1
# #     while current_page<=max_page:
# #         try:
# #             # 等待表格数据加载完成
# #             wait = WebDriverWait(driver, 10)
# #
# #             # 方法1：等待表格行出现
# #             marks = wait.until(
# #                 EC.presence_of_all_elements_located((By.XPATH, '//table//tbody/tr[td[2]/a[contains(@href, "quote.eastmoney.com")]]'))
# #             )
# #             print(f"找到 {len(marks)} 条数据")
# #
# #         except TimeoutException:
# #             print("页面加载超时，未找到数据")
# #             return
# #         for mark in marks:
# #             id = mark.find_element(By.XPATH, './/td[1]').text
# #             code = mark.find_element(By.XPATH, './/td[2]').text
# #             name = mark.find_element(By.XPATH, './/td[3]').text
# #
# #             # 价格信息
# #             last_price = mark.find_element(By.XPATH, './/td[5]').text
# #             change_percent = mark.find_element(By.XPATH, './/td[6]').text
# #             change_amount = mark.find_element(By.XPATH, './/td[7]').text
# #
# #             # 成交量信息
# #             volume = mark.find_element(By.XPATH, './/td[8]').text
# #             turnover = mark.find_element(By.XPATH, './/td[9]').text
# #             turnover_rate = mark.find_element(By.XPATH, './/td[10]').text
# #
# #             # 价格区间
# #             high_price = mark.find_element(By.XPATH, './/td[11]').text
# #             low_price = mark.find_element(By.XPATH, './/td[12]').text
# #             open_price = mark.find_element(By.XPATH, './/td[13]').text
# #             close_price = mark.find_element(By.XPATH, './/td[14]').text
# #             data.append([id,code,name,last_price,change_percent,change_amount,volume,turnover,turnover_rate,high_price,low_price,open_price,close_price])
# #             print(id,code,name,last_price,change_percent,change_amount,volume,turnover,turnover_rate,high_price,low_price,open_price,close_price)
# #             current_page+=1
# #         if current_page < max_page:
# #             if not click_next_page():
# #                 print("没有下一页，停止翻页")
# #                 break
# #
# #         time.sleep(1)  # 翻页间隔
# #         break
# #     return data
# #
# # def click_next_page():
# #     """点击下一页按钮"""
# #     try:
# #         # 查找下一页按钮：包含 > 符号的链接
# #         next_buttons = driver.find_elements(By.XPATH, '//a[contains(text(), ">")]')
# #
# #         for button in next_buttons:
# #             if button.is_displayed() and button.is_enabled():
# #                 button.click()
# #                 time.sleep(2)  # 等待页面加载
# #                 print("成功翻到下一页")
# #                 return True
# #
# #         print("未找到可用的下一页按钮")
# #         return False
# #
# #     except Exception as e:
# #         print(f"翻页失败: {e}")
# #         return False
# #
# #
# # if __name__ == "__main__":
# #     try:
# #         # 获取前3页数据
# #         data = get_html("https://quote.eastmoney.com/center/gridlist.html#sh_a_board", 3)
# #
# #     finally:
# #         input("按回车键关闭浏览器...")
# #         driver.quit()
# from selenium import webdriver
# from selenium.webdriver.common.by import By
# from selenium.webdriver.edge.service import Service  # 导入Edge服务模块
# from beautifultable import BeautifulTable
# from selenium.webdriver.support.ui import WebDriverWait
# from selenium.webdriver.support import expected_conditions as EC
# from selenium.common.exceptions import NoSuchElementException, TimeoutException
# import time
#
# driver = webdriver.Edge(service=Service())
#
#
# def navigate_to_market(market_name):
#     """导航到指定的市场板块"""
#     try:
#         wait = WebDriverWait(driver, 10)
#
#         # 根据市场名称构建XPath
#         if market_name == "沪深京A股":
#             xpath = '//a[contains(text(), "沪深京A股") and contains(@href, "hs_a_board")]'
#         elif market_name == "上证A股":
#             xpath = '//a[contains(text(), "上证A股") and contains(@href, "sh_a_board")]'
#         elif market_name == "深证A股":
#             xpath = '//a[contains(text(), "深证A股") and contains(@href, "sz_a_board")]'
#         else:
#             print(f"不支持的板块: {market_name}")
#             return False
#
#         # 等待元素可点击并点击
#         element = wait.until(EC.element_to_be_clickable((By.XPATH, xpath)))
#         element.click()
#         print(f"成功切换到: {market_name}")
#
#         # 等待页面加载
#         time.sleep(3)
#         return True
#
#     except Exception as e:
#         print(f"切换到 {market_name} 失败: {e}")
#         return False
#
# def get_html(url, max_page,market_name):
#     driver.get(url)
#     data = []
#     current_page = 1
#
#     if not navigate_to_market(market_name):
#         return data
#
#     while current_page <= max_page:
#         try:
#             print(f"\n正在获取第 {current_page} 页数据...")
#
#             # 等待表格数据加载完成
#             wait = WebDriverWait(driver, 10)
#             marks = wait.until(
#                 EC.presence_of_all_elements_located(
#                     (By.XPATH, '//table//tbody/tr[td[2]/a[contains(@href, "quote.eastmoney.com")]]'))
#             )
#             print(f"找到 {len(marks)} 条数据")
#
#             # 提取当前页的所有数据
#             for mark in marks:
#                 try:
#                     id = mark.find_element(By.XPATH, './/td[1]').text
#                     code = mark.find_element(By.XPATH, './/td[2]').text
#                     name = mark.find_element(By.XPATH, './/td[3]').text
#
#                     # 价格信息
#                     last_price = mark.find_element(By.XPATH, './/td[5]').text
#                     change_percent = mark.find_element(By.XPATH, './/td[6]').text
#                     change_amount = mark.find_element(By.XPATH, './/td[7]').text
#
#                     # 成交量信息
#                     volume = mark.find_element(By.XPATH, './/td[8]').text
#                     turnover = mark.find_element(By.XPATH, './/td[9]').text
#                     turnover_rate = mark.find_element(By.XPATH, './/td[10]').text
#
#                     # 价格区间
#                     high_price = mark.find_element(By.XPATH, './/td[11]').text
#                     low_price = mark.find_element(By.XPATH, './/td[12]').text
#                     open_price = mark.find_element(By.XPATH, './/td[13]').text
#                     close_price = mark.find_element(By.XPATH, './/td[14]').text
#
#                     data.append([id, code, name, last_price, change_percent, change_amount,
#                                  volume, turnover, turnover_rate, high_price, low_price,
#                                  open_price, close_price])
#
#                 except NoSuchElementException as e:
#                     print(f"提取数据时元素未找到: {e}")
#                     continue
#
#             # 翻页逻辑（处理完当前页所有数据后再翻页）
#             if current_page < max_page:
#                 if not click_next_page():
#                     print("没有下一页，停止翻页")
#                     break
#
#             # 当前页处理完成，页码递增
#             current_page += 1
#             time.sleep(1)  # 翻页间隔
#
#         except TimeoutException:
#             print("页面加载超时，未找到数据")
#             break
#         except Exception as e:
#             print(f"获取第 {current_page} 页数据时出错: {e}")
#             break
#     print(f"\n总共获取 {len(data)} 条数据")
#     return data
#
# def click_next_page():
#     """使用JavaScript点击下一页按钮"""
#     try:
#         # 查找下一页按钮
#         next_buttons = driver.find_elements(By.XPATH, '//a[contains(text(), ">")]')
#
#         for button in next_buttons:
#             if button.is_displayed() and button.is_enabled():
#                 # 使用JavaScript直接点击，绕过遮罩层
#                 driver.execute_script("arguments[0].click();", button)
#                 time.sleep(3)  # 等待页面加载
#                 print("成功翻到下一页")
#                 return True
#
#         print("未找到可用的下一页按钮")
#         return False
#
#     except Exception as e:
#         print(f"翻页失败: {e}")
#         return False
#
#
#
# if __name__ == "__main__":
#     try:
#         markets=["沪深京A股", "上证A股", "深证A股"]
#         for market_name in markets:
#         # 获取前3页数据
#             data = get_html("https://quote.eastmoney.com/center/gridlist.html#sh_a_board", 2,market_name)
#             print(f"\n数据获取完成，共获取 {len(data)} 条记录")
#             if data:
#                 table = BeautifulTable()
#                 table.set_style(BeautifulTable.STYLE_COMPACT)
#                 table.columns.header = [
#                     '序号', '代码', '名称', '最新价', '涨跌幅', '涨跌额',
#                     '成交量', '成交额', '换手率', '最高价', '最低价', '开盘价', '昨收价'
#                 ]
#                 table.columns.alignment = BeautifulTable.ALIGN_CENTER
#                 table.columns.width = [12,14,12,12,12,12,12,12,12,12,12,12,12]
#
#                 for product in data:  # 只显示前10个作为示例
#                     # 简化商品名称显示
#                     table.rows.append(product)
#                 print(table)
#     finally:
#         input("按回车键关闭浏览器...")
#         driver.quit()
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.service import Service
from beautifultable import BeautifulTable
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import time
import sqlite3
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import sqlite3
import time

driver = webdriver.Edge(service=Service())


class StockDatabase:
    def __init__(self, db_name="stock_data.db"):
        self.db_name = db_name
        self.init_database()

    def init_database(self):
        """初始化数据库和表结构"""
        conn = sqlite3.connect(self.db_name)
        cursor = conn.cursor()

        cursor.execute('''
            CREATE TABLE IF NOT EXISTS stocks (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                serial_number TEXT,
                stock_code TEXT NOT NULL,
                stock_name TEXT NOT NULL,
                last_price REAL,
                change_percent TEXT,
                change_amount REAL,
                volume TEXT,
                turnover TEXT,
                turnover_rate TEXT,
                high_price REAL,
                low_price REAL,
                open_price REAL,
                close_price REAL,
                market_type TEXT,
                crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        ''')

        cursor.execute('CREATE INDEX IF NOT EXISTS idx_stock_code ON stocks(stock_code)')
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_market_type ON stocks(market_type)')
        cursor.execute('CREATE INDEX IF NOT EXISTS idx_crawl_time ON stocks(crawl_time)')

        conn.commit()
        conn.close()
        print(f"数据库初始化完成: {self.db_name}")

    def insert_stock_data(self, stock_data, market_type):
        """插入股票数据"""
        conn = sqlite3.connect(self.db_name)
        cursor = conn.cursor()

        try:
            cursor.execute('''
                INSERT INTO stocks (
                    serial_number, stock_code, stock_name, last_price, change_percent,
                    change_amount, volume, turnover, turnover_rate, high_price,
                    low_price, open_price, close_price, market_type
                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ''', (
                stock_data[0],  # 序号
                stock_data[1],  # 代码
                stock_data[2],  # 名称
                self._convert_to_float(stock_data[3]),  # 最新价
                stock_data[4],  # 涨跌幅
                self._convert_to_float(stock_data[5]),  # 涨跌额
                stock_data[6],  # 成交量
                stock_data[7],  # 成交额
                stock_data[8],  # 换手率
                self._convert_to_float(stock_data[9]),  # 最高价
                self._convert_to_float(stock_data[10]),  # 最低价
                self._convert_to_float(stock_data[11]),  # 开盘价
                self._convert_to_float(stock_data[12]),  # 昨收价
                market_type
            ))

            conn.commit()
            return True

        except Exception as e:
            print(f"插入数据失败: {e}")
            conn.rollback()
            return False
        finally:
            conn.close()

    def _convert_to_float(self, value):
        """将字符串转换为浮点数，处理特殊字符"""
        if not value or value == '-' or value == '--':
            return 0.0
        try:
            cleaned_value = str(value).replace('%', '').replace(',', '').strip()
            return float(cleaned_value)
        except ValueError:
            return 0.0

def get_html(url, max_page):
    """获取HTML数据"""
    driver.get(url)  # 添加这行：先访问URL
    current_page = 1
    data = []

    while current_page <= max_page:
        try:
            print(f"\n正在获取第 {current_page} 页数据...")

            # 等待表格数据加载完成
            wait = WebDriverWait(driver, 10)
            marks = wait.until(
                EC.presence_of_all_elements_located(
                    (By.XPATH, '//table//tbody/tr[td[2]/a[contains(@href, "quote.eastmoney.com")]]'))
            )
            print(f"找到 {len(marks)} 条数据")

            # 提取当前页的所有数据
            for mark in marks:
                try:
                    id = mark.find_element(By.XPATH, './/td[1]').text
                    code = mark.find_element(By.XPATH, './/td[2]').text
                    name = mark.find_element(By.XPATH, './/td[3]').text

                    # 价格信息
                    last_price = mark.find_element(By.XPATH, './/td[5]').text
                    change_percent = mark.find_element(By.XPATH, './/td[6]').text
                    change_amount = mark.find_element(By.XPATH, './/td[7]').text

                    # 成交量信息
                    volume = mark.find_element(By.XPATH, './/td[8]').text
                    turnover = mark.find_element(By.XPATH, './/td[9]').text
                    turnover_rate = mark.find_element(By.XPATH, './/td[10]').text

                    # 价格区间
                    high_price = mark.find_element(By.XPATH, './/td[11]').text
                    low_price = mark.find_element(By.XPATH, './/td[12]').text
                    open_price = mark.find_element(By.XPATH, './/td[13]').text
                    close_price = mark.find_element(By.XPATH, './/td[14]').text

                    data.append([id, code, name, last_price, change_percent, change_amount,
                                 volume, turnover, turnover_rate, high_price, low_price,
                                 open_price, close_price])

                except NoSuchElementException as e:
                    print(f"提取数据时元素未找到: {e}")
                    continue

            # 翻页逻辑（处理完当前页所有数据后再翻页）
            if current_page < max_page:
                if not click_next_page():
                    print("没有下一页，停止翻页")
                    break
            else:
                break  # 已经达到最大页数，退出循环

            # 当前页处理完成，页码递增
            current_page += 1
            time.sleep(2)  # 翻页间隔

        except TimeoutException:
            print("页面加载超时，未找到数据")
            break
        except Exception as e:
            print(f"获取第 {current_page} 页数据时出错: {e}")
            break

    return data  # 添加返回值


def click_next_page():
    """使用JavaScript点击下一页按钮"""
    try:
        # 先等待页面稳定
        time.sleep(1)

        # 查找下一页按钮
        next_buttons = driver.find_elements(By.XPATH, '//a[contains(text(), ">")]')

        for button in next_buttons:
            if button.is_displayed():
                # 使用JavaScript直接点击，绕过遮罩层
                driver.execute_script("arguments[0].click();", button)
                time.sleep(3)  # 等待页面加载

                # 验证是否翻页成功
                wait = WebDriverWait(driver, 5)
                wait.until(EC.presence_of_element_located(
                    (By.XPATH, '//table//tbody/tr[td[2]/a[contains(@href, "quote.eastmoney.com")]]'))
                )
                print("成功翻到下一页")
                return True

        print("未找到可用的下一页按钮")
        return False

    except Exception as e:
        print(f"翻页失败: {e}")
        return False


def main():
    db = StockDatabase()
    try:
        markets = [
            "https://quote.eastmoney.com/center/gridlist.html#hs_a_board",
            "https://quote.eastmoney.com/center/gridlist.html#sh_a_board",
            "https://quote.eastmoney.com/center/gridlist.html#sz_a_board"
        ]

        market_names = ["沪深京A股", "上证A股", "深证A股"]
        all_data = []

        for i, url in enumerate(markets):
            print(f"\n{'=' * 50}")
            print(f"开始处理板块: {market_names[i]}")
            print(f"{'=' * 50}")

            # 修正：直接传递url变量，而不是字符串"url"
            data = get_html(url, 2)

            if data:
                all_data.extend(data)
                print(f"{market_names[i]} 获取到 {len(data)} 条数据")

                # 显示当前板块的数据预览
                print(f"\n{market_names[i]} 数据:")
                table = BeautifulTable()
                table.set_style(BeautifulTable.STYLE_COMPACT)
                table.columns.alignment = BeautifulTable.ALIGN_CENTER
                table.columns.header = [
                    '序号', '代码', '名称', '最新价', '涨跌幅', '涨跌额',
                    '成交量', '成交额', '换手率', '最高价', '最低价', '开盘价', '昨收价'
                ]

                table.columns.width = [12,14,12,12,12,12,12,12,12,12,12,12,12]
                for product in data:  # 只显示前10个作为示例
                    # 简化商品名称显示
                    table.rows.append(product)
                print(table)
            else:
                print(f"{market_names[i]} 没有获取到数据")

            if data:
                inserted_count = 0
                for stock_data in data:
                    if db.insert_stock_data(stock_data, market_names[i]):
                        inserted_count += 1


                print(f"{market_names[i]} 插入 {inserted_count} 条数据")
            else:
                print(f"{market_names[i]} 没有获取到数据")
            # 板块间短暂停顿
            if i < len(markets) - 1:
                time.sleep(3)

    except Exception as e:
        print(f"主程序执行出错: {e}")
    finally:
        input("按回车键关闭浏览器...")
        driver.quit()


if __name__ == "__main__":
    main()